import os
import json
import re
import torch
import argparse
import hashlib
from glob import glob
from tqdm import tqdm
import numpy as np
from collections import defaultdict
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor
from difflib import SequenceMatcher

USE_GPU = torch.cuda.is_available()  # 自动检测GPU是否可用
CPU_THREADS = min(16, os.cpu_count() or 4)  # 自动检测CPU核心数，最大16线程

def load_template_database(file_path):
    """加载现有的模板数据库"""
    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            return json.load(f)
    except FileNotFoundError:
        # 如果文件不存在，返回初始结构
        return {
            "templates": [],
            "site_stats": {},
            "device_stats": {}
        }

def save_template_database(database, file_path):
    """保存更新后的模板数据库"""
    # 确保目标目录存在
    os.makedirs(os.path.dirname(os.path.abspath(file_path)), exist_ok=True)
    with open(file_path, 'w', encoding='utf-8') as f:
        json.dump(database, f, ensure_ascii=False, indent=2)

def extract_timestamp(line):
    """从日志行提取时间戳"""
    timestamp_match = re.search(r'\[(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2})', line)
    if timestamp_match:
        return timestamp_match.group(1)
    return None

def extract_site(line):
    """从日志行提取站点信息"""
    site_match = re.search(r'\[Site:(Site\d+)\]', line)
    if site_match:
        return site_match.group(1)
    return None

def extract_status(line):
    """从日志行提取状态信息(pass/fail/warning)"""
    if "pass" in line.lower():
        return "pass"
    elif "fail" in line.lower():
        return "fail"
    elif "-W]" in line:
        return "warning"
    return None

def normalize_template(line):
    """将日志行转换为模板，替换特定信息为通用标记"""
    # 替换时间戳
    template = re.sub(r'\[\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}-[A-Z]\]', '[<TIMESTAMP>]', line)

    # 替换纯数字序列
    template = re.sub(r'(?<![A-Za-z0-9])\d+(?![A-Za-z0-9])', '<NUM>', template)

    # 替换设备序列号（假设格式类似 "030541FF09010706630031F4"）
    template = re.sub(r'(?<![A-Za-z0-9])[0-9A-F]{24}(?![A-Za-z0-9])', '<DEVICE_SN>', template)

    # 替换十六进制值
    template = re.sub(r'0x[0-9A-F]+', '<HEX>', template)

    return template

def calculate_similarity(template1, template2):
    """计算两个模板的相似度"""
    # 使用序列匹配器计算相似度
    return SequenceMatcher(None, template1, template2).ratio()

def read_log_file(file_path):
    """读取日志文件，支持多种编码和HTML格式"""
    import codecs
    from bs4 import BeautifulSoup

    if not os.path.exists(file_path):
        raise FileNotFoundError(f"日志文件不存在: {file_path}")

    encodings = ['utf-8', 'gbk', 'latin-1', 'gb2312']
    content = None

    for enc in encodings:
        try:
            with codecs.open(file_path, 'r', encoding=enc, errors='replace') as f:
                content = f.read()
            break
        except UnicodeDecodeError:
            continue

    if content is None:
        raise UnicodeError(f"无法使用任何编码读取文件: {file_path}")

    # 处理HTML文件
    if file_path.lower().endswith('.html'):
        try:
            soup = BeautifulSoup(content, 'html.parser')

            log_content = []
            # 尝试从pre标签提取
            pre_tags = soup.find_all('pre')
            if pre_tags:
                for pre in pre_tags:
                    log_content.extend(pre.text.splitlines())

            # 如果没有找到，尝试其他方法
            if not log_content:
                for script in soup(["script", "style"]):
                    script.extract()
                log_content = soup.get_text(separator=' ').splitlines()

            return [line.strip() for line in log_content if line.strip()]
        except Exception as e:
            print(f"解析HTML文件时出错: {e}")
            # 如果HTML解析失败，回退到按行处理
            return [line.strip() for line in content.splitlines() if line.strip()]

    # 非HTML文件直接按行分割
    return [line.strip() for line in content.splitlines() if line.strip()]

def auto_generate_description(template, examples):
    """自动生成模板描述"""
    # 检测日志级别
    log_level = ""
    if "-I]" in template or "INFO" in template:
        log_level = "信息"
    elif "-E]" in template or "ERROR" in template:
        log_level = "错误"
    elif "-W]" in template or "WARN" in template:
        log_level = "警告"
    elif "-D]" in template or "DEBUG" in template:
        log_level = "调试"

    # 识别关键操作词
    operations = {
        "启动": ["start", "启动", "begin", "初始化", "init"],
        "结束": ["end", "完成", "结束", "finish", "complete", "stop"],
        "加载": ["load", "加载", "读取", "read"],
        "保存": ["save", "保存", "写入", "write"],
        "连接": ["connect", "连接", "link"],
        "断开": ["disconnect", "断开", "unlink"],
        "测试": ["test", "测试", "check", "验证", "verify"],
        "配置": ["config", "配置", "setting", "设置", "setup"],
        "更新": ["update", "更新", "refresh"],
        "删除": ["delete", "删除", "remove"],
        "创建": ["create", "创建", "new"],
        "查询": ["query", "查询", "search", "find"]
    }

    operation = ""
    for op, keywords in operations.items():
        if any(keyword in template.lower() for keyword in keywords):
            operation = op
            break

    # 尝试提取模块/组件名称
    module_match = re.search(r'\[(.*?)\]', template)
    module = module_match.group(1) if module_match else ""

    # 生成描述
    if log_level and operation and module:
        return f"{log_level}日志：{module}模块{operation}操作"
    elif log_level and operation:
        return f"{log_level}日志：{operation}操作"
    elif log_level and module:
        return f"{log_level}日志：{module}模块状态记录"
    elif operation and module:
        return f"{module}模块{operation}操作记录"
    elif log_level:
        return f"{log_level}级别日志记录"
    elif operation:
        return f"系统{operation}操作记录"
    elif module:
        return f"{module}模块状态记录"
    else:
        # 使用模板关键词作为描述
        words = re.findall(r'[a-zA-Z]+', template)
        if words:
            significant_words = [w for w in words if len(w) > 3][:3]  # 最多取3个有意义的词
            if significant_words:
                return f"{'、'.join(significant_words)}相关日志记录"

        # 如果以上方法都不奏效，使用通用描述
        return "通用日志记录模板"

def find_best_matching_template(template, templates_dict, similarity_threshold=0.7):
    """查找最匹配的现有模板"""
    best_match = None
    best_similarity = 0

    # 首先尝试精确匹配
    if template in templates_dict:
        return template, 1.0

    # 如果没有精确匹配，寻找最相似的模板
    for existing_template in templates_dict:
        similarity = calculate_similarity(template, existing_template)
        if similarity > best_similarity and similarity >= similarity_threshold:
            best_similarity = similarity
            best_match = existing_template

    return best_match, best_similarity

def merge_similar_templates(templates, similarity_threshold=0.8):
    """合并相似的模板"""
    if not templates:
        return []

    # 按计数排序，优先保留高频模板
    sorted_templates = sorted(templates, key=lambda x: x["count"], reverse=True)

    # 存储已处理的模板，避免重复合并
    processed = set()
    merged_templates = []

    for i, template in enumerate(sorted_templates):
        if i in processed:
            continue

        template_text = template["template"]
        merged_count = template["count"]
        merged_examples = template["examples"].copy()
        description = template.get("description", "")

        # 寻找并合并相似模板
        for j, other_template in enumerate(sorted_templates[i+1:], i+1):
            if j in processed:
                continue

            other_text = other_template["template"]
            similarity = calculate_similarity(template_text, other_text)

            if similarity >= similarity_threshold:
                # 合并这两个模板
                merged_count += other_template["count"]

                # 合并示例，最多保留3个
                for example in other_template["examples"]:
                    if example not in merged_examples and len(merged_examples) < 3:
                        merged_examples.append(example)

                # 如果高频模板没有描述但低频模板有，则使用低频模板的描述
                if not description and other_template.get("description"):
                    description = other_template["description"]

                processed.add(j)

        # 添加合并后的模板
        merged_template = template.copy()
        merged_template["count"] = merged_count
        merged_template["examples"] = merged_examples
        if description:
            merged_template["description"] = description

        merged_templates.append(merged_template)
        processed.add(i)

    # 重新计算百分比
    total_count = sum(t["count"] for t in merged_templates)
    for t in merged_templates:
        percentage = (t["count"] / total_count) * 100 if total_count > 0 else 0
        t["percentage"] = f"{percentage:.2f}%"

    # 重新排序
    return sorted(merged_templates, key=lambda x: x["count"], reverse=True)

def analyze_log_file(file_path, template_db, similarity_threshold=0.7, use_html_parser=True):
    """增量分析日志文件并更新模板数据库"""
    # 提取当前模板字典，用于快速查找
    templates_dict = {t["template"]: t for t in template_db["templates"]}

    # 初始化统计信息
    site_stats = defaultdict(lambda: {"pass": 0, "fail": 0, "warnings": 0, "total": 0})
    device_stats = defaultdict(lambda: {"pass": 0, "fail": 0, "warnings": 0, "total": 0})

    # 从现有数据加载站点和设备统计信息
    for site, stats in template_db.get("site_stats", {}).items():
        site_stats[site].update(stats)

    for year, stats in template_db.get("device_stats", {}).items():
        device_stats[year].update(stats)

    new_templates = []
    template_examples = defaultdict(list)

    # 读取日志文件
    try:
        lines = read_log_file(file_path) if use_html_parser else []
        if not lines and not file_path.lower().endswith('.html'):
            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
                lines = [line.strip() for line in f if line.strip()]
    except Exception as e:
        print(f"读取文件 {file_path} 时出错: {e}")
        return template_db, []

    # 使用进度条处理每一行
    for line in tqdm(lines, desc=f"处理 {os.path.basename(file_path)}", leave=False):
        # 提取信息
        timestamp = extract_timestamp(line)
        site = extract_site(line)
        status = extract_status(line)

        # 更新统计信息
        if site:
            site_stats[site]["total"] += 1
            if status == "pass":
                site_stats[site]["pass"] += 1
            elif status == "fail":
                site_stats[site]["fail"] += 1
            elif status == "warning":
                site_stats[site]["warnings"] += 1

        if timestamp:
            year = timestamp.split('/')[0]
            device_stats[year]["total"] += 1
            if status == "pass":
                device_stats[year]["pass"] += 1
            elif status == "fail":
                device_stats[year]["fail"] += 1
            elif status == "warning":
                device_stats[year]["warnings"] += 1

        # 生成模板
        template = normalize_template(line)

        # 查找最匹配的现有模板
        best_match, similarity = find_best_matching_template(template, templates_dict, similarity_threshold)

        if best_match is not None:
            # 使用现有模板
            template = best_match
        elif template not in templates_dict:
            # 创建新模板
            templates_dict[template] = {
                "template": template,
                "count": 0,
                "percentage": "0.00%",
                "examples": [],
                "description": ""
            }
            new_templates.append(template)

        # 更新模板计数和示例
        templates_dict[template]["count"] += 1

        # 保存示例（最多3个）
        if len(template_examples[template]) < 3 and line not in template_examples[template]:
            template_examples[template].append(line)

    # 更新示例
    for template, examples in template_examples.items():
        if templates_dict[template]["examples"]:
            # 保留原有的例子，只添加新例子
            current_examples = templates_dict[template]["examples"]
            for example in examples:
                if len(current_examples) < 3 and example not in current_examples:
                    current_examples.append(example)
            templates_dict[template]["examples"] = current_examples
        else:
            templates_dict[template]["examples"] = examples

    # 将字典转换回列表
    templates_list = list(templates_dict.values())

    # 计算百分比
    total_lines = sum(t["count"] for t in templates_list)
    for template in templates_list:
        percentage = (template["count"] / total_lines) * 100 if total_lines > 0 else 0
        template["percentage"] = f"{percentage:.2f}%"

    # 按计数排序模板
    sorted_templates = sorted(templates_list, key=lambda x: x["count"], reverse=True)

    # 更新数据库
    template_db["templates"] = sorted_templates
    template_db["site_stats"] = dict(site_stats)
    template_db["device_stats"] = dict(device_stats)

    return template_db, new_templates

def process_log_files(template_db_path, log_files, similarity_threshold=0.7,
                      skip_description=False, parallel=True, max_templates_to_describe=20,
                      auto_describe=True):
    """处理多个日志文件并更新模板数据库，支持并行处理和增量聚类"""
    template_db = load_template_database(template_db_path)

    # 保存原始模板列表，用于后续比较
    original_templates = set(t["template"] for t in template_db.get("templates", []))

    print(f"正在处理 {len(log_files)} 个日志文件...")

    if parallel and len(log_files) > 1:
        # 并行处理多个文件
        merged_db = template_db.copy()

        with ThreadPoolExecutor(max_workers=CPU_THREADS) as executor:
            # 提交所有任务
            futures = []
            for log_file in log_files:
                if not os.path.exists(log_file):
                    print(f"文件不存在: {log_file}")
                    continue

                # 为每个任务创建一个副本
                file_db = {
                    "templates": template_db.get("templates", []).copy(),
                    "site_stats": template_db.get("site_stats", {}).copy(),
                    "device_stats": template_db.get("device_stats", {}).copy()
                }

                future = executor.submit(
                    analyze_log_file,
                    log_file,
                    file_db,
                    similarity_threshold
                )
                futures.append((future, log_file))

            # 初始化合并所需的数据结构
            all_templates = {}
            all_site_stats = defaultdict(lambda: {"pass": 0, "fail": 0, "warnings": 0, "total": 0})
            all_device_stats = defaultdict(lambda: {"pass": 0, "fail": 0, "warnings": 0, "total": 0})
            all_new_templates = []

            # 收集结果并合并
            for future, log_file in tqdm(futures, desc="处理文件"):
                try:
                    file_db, new_templates = future.result()

                    # 合并模板
                    for template in file_db.get("templates", []):
                        template_text = template["template"]
                        if template_text not in all_templates:
                            all_templates[template_text] = template.copy()
                        else:
                            # 累加计数
                            all_templates[template_text]["count"] += template["count"]

                            # 合并示例
                            for example in template["examples"]:
                                if example not in all_templates[template_text]["examples"]:
                                    if len(all_templates[template_text]["examples"]) < 3:
                                        all_templates[template_text]["examples"].append(example)

                    # 合并统计信息
                    for site, stats in file_db.get("site_stats", {}).items():
                        for key, value in stats.items():
                            all_site_stats[site][key] += value

                    for year, stats in file_db.get("device_stats", {}).items():
                        for key, value in stats.items():
                            all_device_stats[year][key] += value

                    # 收集新模板
                    all_new_templates.extend(new_templates)

                    print(f"✓ 已完成 {log_file} 分析")
                except Exception as e:
                    print(f"✗ 处理 {log_file} 时出错: {e}")

            # 合并相似模板（进一步聚类）
            merged_templates = merge_similar_templates(list(all_templates.values()), similarity_threshold)

            # 更新最终数据库
            merged_db["templates"] = merged_templates
            merged_db["site_stats"] = dict(all_site_stats)
            merged_db["device_stats"] = dict(all_device_stats)

            # 找出真正的新模板
            final_templates = set(t["template"] for t in merged_db["templates"])
            actual_new_templates = final_templates - original_templates

            print(f"实际新增模板数: {len(actual_new_templates)}")

            # 保存更新后的模板数据库
            save_template_database(merged_db, template_db_path)
            print(f"已保存更新后的模板数据库到: {template_db_path}")

            # 为新模板添加描述
            if actual_new_templates and not skip_description:
                # 按计数排序模板，仅处理高频模板
                templates_to_describe = []
                for template_text in actual_new_templates:
                    for t in merged_db["templates"]:
                        if t["template"] == template_text:
                            templates_to_describe.append((t, t["count"]))
                            break

                templates_to_describe.sort(key=lambda x: x[1], reverse=True)
                templates_to_describe = templates_to_describe[:max_templates_to_describe]

                print(f"\n需要描述的高频新模板 ({len(templates_to_describe)}/{len(actual_new_templates)}):")

                if auto_describe:
                    print("\n正在自动生成描述...")
                    for template_obj, count in templates_to_describe:
                        template_text = template_obj["template"]
                        examples = template_obj.get("examples", [])

                        # 自动生成描述
                        auto_desc = auto_generate_description(template_text, examples)
                        template_obj["description"] = auto_desc

                    print("已完成自动描述")
                else:
                    # 批量操作选项
                    print("\n[操作选项] 输入 'skip' 跳过所有, 'auto' 自动生成描述")
                    choice = input("请选择操作或按回车继续手动描述: ")

                    if choice.lower() == 'skip':
                        print("已跳过所有描述")
                    elif choice.lower() == 'auto':
                        print("正在自动生成描述...")
                        for template_obj, count in templates_to_describe:
                            template_text = template_obj["template"]
                            examples = template_obj.get("examples", [])

                            # 自动生成描述
                            auto_desc = auto_generate_description(template_text, examples)
                            template_obj["description"] = auto_desc

                        print("已完成自动描述")
                    else:
                        for template_obj, count in templates_to_describe:
                            template_text = template_obj["template"]
                            print(f"\n模板 (出现{count}次): {template_text}")
                            description = input("请输入描述 (或按回车跳过): ")
                            if description:
                                template_obj["description"] = description

                # 保存带有描述的更新数据库
                save_template_database(merged_db, template_db_path)
                print("已保存带描述的模板数据库")

            return merged_db
    else:
        # 串行处理
        for log_file in tqdm(log_files, desc="处理文件"):
            if not os.path.exists(log_file):
                print(f"文件不存在: {log_file}")
                continue

            template_db, new_templates = analyze_log_file(
                log_file,
                template_db,
                similarity_threshold
            )
            print(f"已分析 {log_file}")

        # 找出真正的新模板
        final_templates = set(t["template"] for t in template_db["templates"])
        actual_new_templates = final_templates - original_templates
        print(f"实际新增模板数: {len(actual_new_templates)}")

        # 保存更新后的模板数据库
        save_template_database(template_db, template_db_path)
        print(f"已保存更新后的模板数据库到: {template_db_path}")

        # 自动为新模板添加描述
        if actual_new_templates and not skip_description:
            # 按计数排序模板
            templates_to_describe = []
            for template_text in actual_new_templates:
                for t in template_db["templates"]:
                    if t["template"] == template_text:
                        templates_to_describe.append((t, t["count"]))
                        break

            templates_to_describe.sort(key=lambda x: x[1], reverse=True)
            templates_to_describe = templates_to_describe[:max_templates_to_describe]

            print(f"\n需要描述的高频新模板 ({len(templates_to_describe)}/{len(actual_new_templates)}):")

            if auto_describe:
                print("\n正在自动生成描述...")
                for template_obj, count in templates_to_describe:
                    template_text = template_obj["template"]
                    examples = template_obj.get("examples", [])

                    # 自动生成描述
                    auto_desc = auto_generate_description(template_text, examples)
                    template_obj["description"] = auto_desc

                print("已完成自动描述")

                # 保存带有描述的更新数据库
                save_template_database(template_db, template_db_path)
                print("已保存带描述的模板数据库")

        return template_db

def find_log_files(base_dirs, patterns=None, recursive=True):
    """查找符合条件的日志文件"""
    if not patterns:
        patterns = ['*.log', '*.txt', '*.html']

    all_files = []

    for base_dir in base_dirs:
        if os.path.isfile(base_dir):
            all_files.append(base_dir)
            continue

        for pattern in patterns:
            if recursive:
                # 递归搜索所有子目录
                search_pattern = os.path.join(base_dir, '**', pattern)
                found_files = glob(search_pattern, recursive=True)
            else:
                # 只搜索当前目录
                search_pattern = os.path.join(base_dir, pattern)
                found_files = glob(search_pattern)

            all_files.extend(found_files)

    return sorted(list(set(all_files)))  # 去重并排序

def main():
    """主函数，支持命令行参数"""
    parser = argparse.ArgumentParser(description='批量处理日志文件并生成模板库')
    parser.add_argument('--db', default="log_templates.json", help='模板数据库文件路径')
    parser.add_argument('--logs', nargs='+', help='日志文件或目录路径列表')
    parser.add_argument('--pattern', nargs='+', default=['*.log', '*.txt', '*.html'],
                        help='日志文件名匹配模式，默认为*.log, *.txt, *.html')
    parser.add_argument('--recursive', action='store_true', help='是否递归搜索子目录')
    parser.add_argument('--skip-desc', action='store_true', help='跳过添加模板描述步骤')
    parser.add_argument('--sequential', action='store_true', help='使用顺序处理而非并行')
    parser.add_argument('--similarity', type=float, default=0.7, help='模板相似度阈值(0-1)')
    parser.add_argument('--auto-describe', action='store_true', help='自动生成模板描述')
    parser.add_argument('--max-templates', type=int, default=20, help='最多处理的高频模板数量')

    args = parser.parse_args()

    # 默认日志文件列表
    default_logs = [
        r'\\172.27.41.81\acroview\MultiAprog\log\2025-08-27\log[20250827-045054].html',
        r'\\172.27.41.81\acroview\MultiAprog\log\2025-08-27\log[20250827-171801].html',
        r'\\172.27.41.81\acroview\MultiAprog\log\2025-08-26\log[20250826-053101].html',
        r'\\172.27.41.81\acroview\MultiAprog\log\2025-08-26\log[20250826-160030].html',
        r'\\172.27.41.81\acroview\MultiAprog\log\2025-08-26\log[20250826-160920].html',
        r'\\172.27.41.81\acroview\MultiAprog\log\2025-08-25\log[20250825-214429].html'
    ]

    # 使用命令行参数或默认值
    log_paths = args.logs if args.logs else default_logs

    # 查找所有符合条件的日志文件
    log_files = find_log_files(log_paths, args.pattern, args.recursive)
    print(f"共找到 {len(log_files)} 个日志文件")

    if not log_files:
        print("未找到任何日志文件，退出程序")
        return

    # 使用GPU加速处理（如果可用）
    if USE_GPU:
        print(f"使用GPU加速处理: {torch.cuda.get_device_name(0)}")
    else:
        print(f"使用CPU处理，线程数: {CPU_THREADS}")

    # 处理日志文件
    updated_db = process_log_files(
        args.db,
        log_files,
        similarity_threshold=args.similarity,
        skip_description=args.skip_desc,
        parallel=not args.sequential,
        max_templates_to_describe=args.max_templates,
        auto_describe=args.auto_describe
    )

    print(f"分析完成，共有 {len(updated_db['templates'])} 个模板")

    # 打印站点统计信息
    if updated_db.get("site_stats"):
        print("\n站点测试统计:")
        for site, stats in updated_db["site_stats"].items():
            pass_rate = stats["pass"] / max(1, stats["pass"] + stats["fail"]) * 100
            print(f"站点 {site}: 通过 {stats['pass']}个, 失败 {stats['fail']}个, "
                  f"通过率 {pass_rate:.1f}%, 警告 {stats['warnings']}个")

if __name__ == "__main__":
    main()
